package bot;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.zip.Checksum;

import datacollection.WebParser;

public class PageScanner {
	
	private static final File from = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\GeoSentenceSearch.txt");
	private static final File one = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseOne.txt");
	private static final File two = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseTwo.txt");
	private static final File three = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseThree.txt");
	private static final File four = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseFour.txt");
	private static final File five = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseFive.txt");
	private static final File six = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseSix.txt");
	private static final File seven = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseSeven.txt");
	private static final File eight = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseEight.txt");
	private static final File nine = new File("c:\\Users\\Jamison\\Documents\\PopulationInfo\\ParseNine.txt");
	private static int start = 0;
	private static String refDate = "2013-01-25";
	

	public static void main(String[] args){
		List<String> nullLines = new ArrayList<String>();
		try {
			WebParser web = new WebParser();
			
			BufferedReader input = new BufferedReader(new FileReader(from));
			FileOutputStream writerOne = new FileOutputStream(one, true);
			FileOutputStream writerTwo = new FileOutputStream(two, true);
			FileOutputStream writerThree = new FileOutputStream(three, true);
			FileOutputStream writerFour = new FileOutputStream(four, true);
			FileOutputStream writerFive = new FileOutputStream(five, true);
			FileOutputStream writerSix = new FileOutputStream(six, true);
			FileOutputStream writerSeven = new FileOutputStream(seven, true);
			FileOutputStream writerEight = new FileOutputStream(eight, true);
			FileOutputStream writerNine = new FileOutputStream(nine, true);
			String line = input.readLine();
			int i = 0;
			String newLine = "\n";
			while (line != null){
				try{
					if (i >= start){
						String source = web.getWikiEditSource(line);
						boolean result = estimatedPopSentence(source, writerOne);
						if (result){
							writerOne.write(line.getBytes());
							writerOne.write(newLine.getBytes());
						}
						boolean resultTwo = checkLeftOverSentence(source, writerTwo);
						if (resultTwo){
							writerTwo.write(line.getBytes());
							writerTwo.write(newLine.getBytes());
						}
						boolean resultThree = checkAreaAfterDemo(source, writerThree);
						if (resultThree){
							writerThree.write(line.getBytes());
							writerThree.write(newLine.getBytes());
						}
						boolean resultFour = extraParagraphsIn2000Info(source, writerFour);
						if (resultFour){
							writerFour.write(line.getBytes());
							writerFour.write(newLine.getBytes());
						}
						boolean resultFive = populationAsOfBug(source);
						if (resultFive){
							writerFive.write(line.getBytes());
							writerFive.write(newLine.getBytes());
						}
						boolean resultSix = duplicated2010Sentence(source, writerSix);
						if (resultSix){
							writerSix.write(line.getBytes());
							writerSix.write(newLine.getBytes());
						}
						boolean resultSeven = junkAfterFields(source, writerSeven);
						if (resultSeven){
							writerSeven.write(line.getBytes());
							writerSeven.write(newLine.getBytes());
						}
						boolean resultEight = textAfterPop(source, writerEight);
						if (resultEight){
							writerEight.write(line.getBytes());
							writerEight.write(newLine.getBytes());
						}
						boolean resultNine = brokenHeader(source, writerNine);
						if (resultNine){
							writerNine.write(line.getBytes());
							writerNine.write(newLine.getBytes());
						}
					}
				}
				catch (NullPointerException e){
					nullLines.add(line);
				}
				System.out.println(i);
				line = input.readLine();
				i++;
//				if (i == 30){
//					System.exit(0);
//				}
			}
		} 
		catch (FileNotFoundException e) {
			e.printStackTrace();
		} 
		catch (IOException e) {
			e.printStackTrace();
		}
		System.out.println("NULL LINES");
		for (String str: nullLines){
			System.out.println(str);
		}
	}
	
	private static boolean checkWasWas(String source){
		PageGenerator page = new PageGenerator();
		try {
			String demoSection = (String) page.getDemoSection(source).get(0);
			if (demoSection.split("was was").length > 1){
				return true;
			}
			else{
				return false;
			}
		} 
		catch (AutomationException e) {
			return true;
		}
	}
	
	private static boolean checkPacificIslanders(String source){
		if (source.split("\\[\\[Race \\(U\\.S\\. Census\\)\\|Pacific Islander\\]\\]").length > 1){
			return true;
		}
		return false;
	}
	
	private static boolean checkFormat(String source){
		if (source.split("<!-- automatically").length == 2){
			return true;
		}
		return false;
	}
	
	/**
	 * Checks for a left over text after the geography sentence 
	 */
	private static boolean checkLeftOverSentence(String source, FileOutputStream writer){
		//find geo section
		String openings[] = {"==Geography==",
						     "== Geography ==",
						     "==Geography and climate==",
						     "== Geography and climate =="
						    };
		
		String test[] = null;
		for (String str: openings){
			test = source.split(str);
			if (test.length == 2){
				break;
			}
		}
		//if no geo section was found return false
		if (test == null){
			return false;
		}
		String geoSection = null;
		try{
			geoSection = test[1].split("==")[0];
		}
		catch (ArrayIndexOutOfBoundsException e){
			String output = "JUST LOOK AT THIS ONE IT BROKE THE CODE";
			try {
				writer.write(output.getBytes());
				return true;
			} 
			catch (IOException e1){
				e1.printStackTrace();
			}
		}
		String regex = "2=United States Census Bureau\\}\\}\\|accessdate=" + refDate + "\\}\\}</ref>";
		String toAnalyze = null;
		try{
			toAnalyze = geoSection.split(regex)[1];
		}
		catch (ArrayIndexOutOfBoundsException e){
			try{
				String toWrite = "~~~~~~~~~~~~~~ArrayIndexOutOfBoundsException~~~~~~~~~~~~~\n" + geoSection;
				writer.write(toWrite.getBytes());
			} 
			catch (IOException e1){
				e1.printStackTrace();
			}
			return true;
		}
		for (int i = 0; i < toAnalyze.length(); i++){
			if (toAnalyze.charAt(i) != ' ' && toAnalyze.charAt(i) != '\n'){
				try{
					writer.write(toAnalyze.getBytes());
				} 
				catch (IOException e){
					e.printStackTrace();
				}
				return true;
			}
		}
		
		
		return false;
	}
	
	/**
	 * Makes sure that the 2010 header has new line in front of it 
	 */
	private static boolean brokenHeader(String source, FileOutputStream writer){
		String test[] = source.split("=== *?2010 census *?===");
		if (test.length != 2){
			try{
				String output = "Header not found\n";
				writer.write(output.getBytes());
			}
			catch (IOException e){
				e.printStackTrace();
			}
			return true;
		}
		else{
			if (test[0].charAt(test[0].length() - 1) == '\n'){
				return false;
			}
		}
		return true;
	}
	
	/**
	 *  Checks for left over text above the 2010 header so that duplicated census info can be found
	 */
	private static boolean checkAreaAfterDemo(String source, FileOutputStream writer){
		try{
			String demoHeaders[] = {"==Demographics==",
					"== Demographics =="
			};
			String test[] = null;
			for (String str: demoHeaders){
				test = source.split(str);
				if (test.length == 2){
					break;
				}
			}
			if (test.length == 1){
				String output = "COULDN'T FIND DEMOGRAPHICS SECTION\n";
				writer.write(output.getBytes());
				return true;
			}
			
			test = test[1].split("===2010 (c|C)ensus===");
			if (test.length != 2){
				String output = "COULDN'T FIND 2010 HEADER\n";
				writer.write(output.getBytes());
				return true;
			}
			
			int openings = 0;
			int end = 0;
			StringBuilder builder = new StringBuilder(100);
			for (int i = 0; i < test[0].length() - 1; i++){
				if (test[0].substring(i, i + 2).equals("{{")){
					openings++;
					if (openings == 1){
						builder.append(test[0].substring(end, i));
					}
				}
				else if (test[0].substring(i, i + 2).equals("}}")){
					openings--;
					if (openings == 0){
						end = i + 2;
					}
				}
			}
			builder.append(test[0].substring(end, test[0].length()));
			String text = builder.toString();
			boolean spaceNewLineTest = true;
			for (int i = 0; i < text.length(); i++){
				if (text.charAt(i) != ' ' && text.charAt(i) != '\n'){
					spaceNewLineTest = false;
					break;
				}
			}
			if (!builder.toString().equals("") && !spaceNewLineTest){
				writer.write(builder.toString().getBytes());
				return true;
			}
		}
		catch (IOException e){
			e.printStackTrace();
		}
				
				
		return false;
	}
	
	private static boolean estimatedPopSentence(String source, FileOutputStream writer){
		String opening = null;
		try{
			String topHalf = source.split("==")[0];
			opening = topHalf.split("'''")[2];
		}
		catch (IndexOutOfBoundsException e){
			String output = "REGEX FAILURE\n";
			try{
				writer.write(output.getBytes());
			}
			catch (IOException e1){
				e1.printStackTrace();
			}
			return true;
		}
		String test[] = opening.split("estimate");
		if (test.length > 1){
			try{
				writer.write(opening.getBytes());
				return true;
			}
			catch (IOException e){
				e.printStackTrace();
			}
		}
		return false;
	}
	
	private static boolean populationAsOfBug(String source){
		String test[] = source.split("\\| *?population_as_of *?= *?2011\n");
		if (test.length == 2){
			return false;
		}
		test = source.split("\\|population_as_of         = \\[\\[2010 United States Census\\|2010\\]\\]\n");
		if (test.length != 2){
			return true;
		}
		return false;
	}
	
	private static boolean brokenReference(String source){
		String parts[] = source.split("==");
		String refClose[] = parts[0].split("</ref>");
		for (int i = 0; i < refClose.length - 1; i++){
			for (int j = refClose[i].length() - 1; j >= 0; j--){
				if (refClose[i].charAt(j) == '\n'){
					return true;
				}
				else if (refClose[i].charAt(j) == '<'){
					if (j + 4 <= refClose[i].length()){
						if (refClose[i].substring(j, j + 4).equals("<ref")){
							break;
						}
					}
				}
			}
		}
		return false;
	}
	
	private static boolean duplicated2010Sentence(String source, FileOutputStream writer){
		try{
			String infobox = Util.getInfobox(source);
			String noInfobox = source.substring(infobox.length());
			String opening = noInfobox.split("==")[0];
			String mySplit[] = opening.split("\\[\\[2010 United States Census\\|2010 census\\]\\]");
			if (mySplit.length == 2){
				String firstHalf[] = Util.removeReferences(mySplit[0]).split("2010");
				String secondHalf[] = Util.removeReferences(mySplit[1]).split("2010");
				if (firstHalf.length > 1 || secondHalf.length > 1){
					return true;
				}
			}
		}
		catch (IndexOutOfBoundsException e){
			String output = "INDEX OUT OF BOUNDS";
			try{
				writer.write(output.getBytes());
			} 
			catch (IOException e1){
				e1.printStackTrace();
			}
			return true;
		}
		catch (IllegalArgumentException e){
			//not infobox doesn't matter
			return false;
		}
		return false;
	}
	
	private static boolean junkAfterFields(String source, FileOutputStream writer){
		String test[] = source.split("\\| *?population_as_of *?= *?2011\n");
		if (test.length == 2){
			return false;
		}
		
		String infobox = null;
		try{
			infobox = Util.getInfobox(source);
		}
		catch (IllegalArgumentException e){
			String output = "No infobox\n";
			try{
				writer.write(output.getBytes());
			}
			catch (IOException e1){
				e1.printStackTrace();
			}
			return true;
		}
		String popDensityMile[] = infobox.split("\\| *?population_density_sq_mi *?= *?[\\p{Digit},\\.]++ *?\n");
		String popDensityKm[] = infobox.split("\\| *?population_density_km2 *?= *?[\\p{Digit},\\.]++ *?\n");
		String areaTotalKm[] = infobox.split("\\| *?area_total_km2 *?= *?[\\p{Digit},\\.]++ *?\n");
		String areaLandKm[] = infobox.split("\\| *?area_land_km2 *?= *?[\\p{Digit},\\.]++ *?\n");
		String areaWaterKm[] = infobox.split("\\| *?area_water_km2 *?= *?[\\p{Digit},\\.]++ *?\n");
		String areaTotalMi[] = infobox.split("\\| *?area_total_sq_mi *?= *?[\\p{Digit},\\.]++ *?\n");
		String areaLandMi[] = infobox.split("\\| *?area_land_sq_mi *?= *?[\\p{Digit},\\.]++ *?\n");
		String areaWaterMi[] = infobox.split("\\| *?area_water_sq_mi *?= *?[\\p{Digit},\\.]++ *?\n");
		int total = areaWaterMi.length + areaLandMi.length + areaTotalMi.length + areaWaterKm.length + areaLandKm.length + areaTotalKm.length
					+ popDensityKm.length + popDensityMile.length;
		if (total != 16){
			return true;
		}
		return false;
	}
	
	private static boolean percentageCheck(String source){
		String regex[] = {"\\| *?area_water_percent *?= *?[\\p{Digit},\\.]++",
						  "\\| *?area_land_percent *?= *?[\\p{Digit},\\.]++"
		};
		for (String str: regex){
			if (source.split(str).length == 2){
				return true;
			}
		}
		return false;
	}
	
	private static boolean doublePopDensity(String source){
		String test[] = source.split("population_density_km2");
		String test2[] = source.split("population_density_sq_mi");
		
		if (test.length != 2 || test2.length != 2){
			return true;
		}
		return false;
	}

	private static boolean extraParagraphsIn2000Info(String source, FileOutputStream writer){
		try{
			String test[] = source.split("===2000 census===");
			if (test.length == 2){
				test = test[1].split("==");
				if (test.length >= 2){
					boolean startCounting = false;
					int newLineCount = 0;
					for (int i = test[0].length() - 1; i >= 0; i--){
						if (startCounting && test[0].charAt(i) == '\n'){
							newLineCount++;
						}
						else if (!startCounting && test[0].charAt(i) != '\n'){
							startCounting = true;
						}
					}
					if (newLineCount > 7){
						writer.write(test[0].getBytes());
						return true;
					}
					else{
						if (test[0].split("2010").length > 1){
							writer.write(test[0].getBytes());
							return true;
						}
					}
				}
				else{
					String output = "COULDN'T FIND ENDING";
					writer.write(output.getBytes());
					return true;
				}
			}
		}
		catch (IOException e){
			e.printStackTrace();
		}
		return false;
	}
	
	private static boolean textAfterPop(String source, FileOutputStream writer){
		try{
			String test[] = source.split("\\| *?population_total *?= *?[\\p{Digit},\\.]++");
			if (test.length != 2){
				String output = "Too many population_totals";
				writer.write(output.getBytes());
				return true;
			}
			else{
				if (test[1].charAt(0) != '\n'){
					String output = "";
					for (int i = 0; i < test[1].length(); i++){
						output += test[1].charAt(i);
						if (test[1].charAt(i) == '\n'){
							writer.write(output.getBytes());
							return true;
						}
					}
				}
			}
		}
		catch (IOException e){
			e.printStackTrace();
		}
		return false;
	}
	
	private static boolean commentBug(String str){
		String test[] = str.split("area_total_km2");
		if (test.length > 2){
			return true;
		}
		return false;
	}
}

